let rec lex = parser
   (* whitespace *)
   | [< ' (' ' | '\n' | '\r' | '\t'); stream >] -> lex stream

   (* slash *)
   | [< ' ('/'); stream >] -> lex_slash stream

   (* zero *)
   | [< ' ('0') as c; stream >] ->
      lex_buffer c lex_zero stream

   (* number *)
   | [< ' ('1'..'9') as c; stream >] ->
      lex_buffer c lex_number stream

   (* string *)
   | [< ' ('"') as c; stream >] ->
      lex_buffer c lex_string stream

   (* identifier *)
   | [< ' ('_' | 'A'..'Z' | 'a'..'z' as c); stream >] ->
      lex_buffer c lex_identifier stream

   (* symbol *)
   | [< ' ('!' | '#'..'/' | ':'..'@' | '['..'`' | '{'..'~' as c); stream >] ->
      lex_buffer c lex_symbol stream

   (* unknown character *)
   | [< 'c; stream >] -> [< 'Token.Unexpected_char c; lex stream >]

   (* end of stream *)
   | [< >] -> [< >]

(* creates a buffer with c in it and continues parsing with function f *)
and lex_buffer c f stream =
   let buffer = Buffer.create 1 in
      Buffer.add_char buffer c;
      f buffer stream

(* starting with a slash can be a symbol or a comment *)
and lex_slash = parser
   | [< ' ('/'); stream >] -> lex_line_comment stream
   | [< ' ('*'); stream >] -> lex_nested_comment 0 stream
   | [< stream >] ->
      lex_buffer '/' lex_symbol stream

(* line comment skips everything until the next newline *)
and lex_line_comment = parser
   | [< ' ('\n'); stream = lex >] -> stream
   | [< 'c; stream = lex_line_comment >] -> stream
   | [< >] -> [< >]

(* nested comment *)
and lex_nested_comment n = parser
   | [< ' ('/'); stream >] -> lex_nested_comment_slash n stream
   | [< ' ('*'); stream >] -> lex_nested_comment_star n stream
   | [< 'c; stream >] -> lex_nested_comment n stream
   | [< >] ->
      [< 'Token.Unexpected_eof >]

and lex_nested_comment_slash n = parser
   | [< ' ('*'); stream >] -> lex_nested_comment (n + 1) stream
   | [< 'c; stream >] -> lex_nested_comment n stream
   | [< >] ->
      [< 'Token.Unexpected_eof >]

and lex_nested_comment_star n = parser
   | [< ' ('/'); stream >] ->
      if n > 0 then
         lex_nested_comment (n - 1) stream
      else
         lex stream
   | [< 'c; stream >] -> lex_nested_comment n stream
   | [< >] ->
      [< 'Token.Unexpected_eof >]

(* starting with a zero can be binary or hex as well as integer, real or imaginary *)
and lex_zero buffer = parser
   | [< ' ('b' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_binary buffer stream
   | [< ' ('x' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_hex buffer stream
   | [< ' ('0'..'9' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_number buffer stream
   | [< ' ('.' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_dot buffer stream
   | [< ' ('e' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_e buffer stream
   | [< ' ('i'); stream = lex >] ->
      [< 'Token.Imaginary (float_of_string (Buffer.contents buffer)); stream >]
   | [< stream = lex >] ->
      [< 'Token.Integer (int_of_string (Buffer.contents buffer)); stream >]

(* binary is 0b[0..1], with arbitrary _ separators *)
and lex_binary buffer = parser
   | [< ' ('_'); stream >] -> lex_binary buffer stream
   | [< ' ('0'..'1' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_binary buffer stream
   | [< stream = lex >] ->
      [< 'Token.Integer (int_of_string (Buffer.contents buffer)); stream >]

(* hex is 0x[0..9 | A..F], with arbitrary _ separators *)
and lex_hex buffer = parser
   | [< ' ('_'); stream >] -> lex_binary buffer stream
   | [< ' ('0'..'9' | 'A'..'F' | 'a'..'f') as c; stream >] ->
      Buffer.add_char buffer c;
      lex_hex buffer stream
   | [< stream = lex >] ->
      [< 'Token.Integer (int_of_string (Buffer.contents buffer)); stream >]

(* so far we're an integer, might change to a real or imaginary *)
and lex_number buffer = parser
   | [< ' ('0'..'9' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_number buffer stream
   | [< ' ('.' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_dot buffer stream
   | [< ' ('e' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_e buffer stream
   | [< ' ('i'); stream = lex >] ->
      [< 'Token.Imaginary (float_of_string (Buffer.contents buffer)); stream >]
   | [< stream = lex >] ->
      [< 'Token.Integer (int_of_string (Buffer.contents buffer)); stream >]

(* so far we're a real with a decimal point, might have an exponent or be imaginary *)
and lex_real_dot buffer = parser
   | [< ' ('0'..'9' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_dot buffer stream
   | [< ' ('e' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_e buffer stream
   | [< ' ('i'); stream = lex >] ->
      [< 'Token.Imaginary (float_of_string (Buffer.contents buffer)); stream >]
   | [< stream = lex >] ->
      [< 'Token.Real (float_of_string (Buffer.contents buffer)); stream >]

(* so far we're a real with an exponent, might be imaginary *)
and lex_real_e buffer = parser
   | [< ' ('0'..'9' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_post_e buffer stream
   | [< ' ('+' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_post_e buffer stream
   | [< ' ('-' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_post_e buffer stream
   | [< ' ('i'); stream = lex >] ->
      [< 'Token.Imaginary (float_of_string (Buffer.contents buffer)); stream >]
   | [< stream = lex >] ->
      [< 'Token.Real (float_of_string (Buffer.contents buffer)); stream >]

(* so far we're a real with an exponent and a + or - or digit, might be imaginary *)
and lex_real_post_e buffer = parser
   | [< ' ('0'..'9' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_real_post_e buffer stream
   | [< ' ('i'); stream = lex >] ->
      [< 'Token.Imaginary (float_of_string (Buffer.contents buffer)); stream >]
   | [< stream = lex >] ->
      [< 'Token.Real (float_of_string (Buffer.contents buffer)); stream >]

(* string is any data enclosed in quotes *)
and lex_string buffer = parser
   | [< ' ('"') as c; stream = lex >] ->
      Buffer.add_char buffer c;
      [< 'Token.String (Buffer.contents buffer); stream >]
   | [< 'c; stream >] ->
      Buffer.add_char buffer c;
      lex_string buffer stream
   | [< >] ->
      [< 'Token.Unexpected_eof >]

(* valid identifier *)
and lex_identifier buffer = parser
   | [< ' ('_' | 'A'..'Z' | 'a'..'z' | '0'..'9' as c); stream >] ->
      Buffer.add_char buffer c;
      lex_identifier buffer stream
   | [< stream = lex >] ->
      [< 'Token.get_identifier (Buffer.contents buffer); stream >]

(* valid symbol *)
and lex_symbol buffer = parser
   | [< ' ('!' | '#'..'/' | ':'..'@' | '['..'`' | '{'..'~' as c); stream >] ->
      let base_sym = Buffer.contents buffer in
         Buffer.add_char buffer c;
         let sym = Buffer.contents buffer in
            if Token.is_symbol_fragment sym then
               lex_symbol buffer stream
            else
               [< 'Token.get_symbol base_sym; lex_buffer c lex_symbol stream >]
   | [< stream = lex >] ->
      [< 'Token.get_symbol (Buffer.contents buffer); stream >]
